# 知识点
# 1.在函数内部对全局变量的修改后，在函数执行完毕，修改的结果是无效的，全局变量并不会受到影响
# 2.filter()
# 3.split()
# 4.replace()


'''
    0 = '&#x9fa4'
    1 = '&#x9f92'
    2 = '&#x9ea3'
    3 = '&#x9a4b'
    4 = '&#x993c'
    5 = '&#x9fa5'
    6 = '&#x9476'
    7 = '&#x958f'
    8 = '&#x9f64'
    9 = '&#x9e3a'
'''
import re
import json
import requests
import time
from requests.exceptions import RequestException

# item是字符串


def datatransform(item):
    if '&#x9f92' in item:
        item = item.replace('&#x9f92', '0')
    elif '&#x9fa4' in item:  # replace()方法不会改变原string的内容！！！
        item = item.replace('&#x9fa4', '1')
    elif '&#x9ea3' in item:
        item = item.replace('&#x9ea3', '2')
    elif '&#x9a4b' in item:
        item = item.replace('&#x9a4b', '3')
    elif '&#x958f' in item:
        item = item.replace('&#x958f', '4')
    elif '&#x9476' in item:
        item = item.replace('&#x9476', '5')
    elif '&#x9f64' in item:
        item = item.replace('&#x9f64', '6')
    elif '&#x9fa5' in item:
        item = item.replace('&#x9fa5', '7')
    elif '&#x993c' in item:
        item = item.replace('&#x993c', '8')
    elif '&#x9e3a' in item:
        item = item.replace('&#x9e3a', '9')
    # print(item)
    return item


def write(content):
    with open("./result-58.txt", 'a', encoding='utf-8') as f:
        f.write(json.dumps(content, ensure_ascii=False) + "\n")


# 爬取某一页函数源代码
def getPage(url):

    try:
        # 设置请求信息，得到返回对象
        headers = {'User-Agent': 'User-Agent:Mozilla/5.0(WindowsNT6.1;rv:2.0.1)Gecko/20100101Firefox/4.0.1'}
        data = {}
        res = requests.get(url, params=data, headers=headers)
        # 判断是否获得响应信息
        if res.status_code == 200:
            return res.content.decode('utf-8')
        else:
            return None
    except RequestException:
        return None

# 数据提取并且处理数据格式


def analysisPage(html):
    # 声明全局变量
    global titlelist_1, titlelist_2, titlelist_real, piclist_real, roomformlist, roomformlist_real, arealist, arealist_real
    # 标题正则
    titlepat = 'target="_blank"  rel="nofollow" >([\\s\\S]*?)</a>'
    titlepat2 = 'target="_blank" >[\\s\\S](.*?)</a>'
    # 图片正则
    picpat = 'lazy_src="(.*?)"'
    # 房子户型正则
    roomformpat = '<p class="room strongbox">(.*?)\s\S{1,}'
    # 房子面积正则`
    roompat = '<p class="room strongbox">(.*?)\s{1,}&nbsp;&nbsp;&nbsp;&nbsp;(.*?);㎡</p>'
    # 价格正则
    pricepat = '<b class="strongbox">(.*?);(.*?);(.*?);(.*?);</b>'

    titlelist = re.findall(titlepat, html)
    titlelist2 = re.findall(titlepat2, html)
    titlelistall = titlelist + titlelist2
    piclist_real = re.findall(picpat, html)
    piclist_real.pop()
    pricelist = re.findall(pricepat, html)
    roomlist = re.findall(roompat, html)
    roomformlist = re.findall(roomformpat, html)
    # 处理标题信息
    titlelist_1 = []  # 用于接收处理掉空格猴的标题数据列表
    titlelist_2 = []
    titlelist_real = []  # 用于接收完全处理好的标题数据列表


# 负责从整个标题数据列表中提取单条标题信息，并进行去空格和切割处理
    for item in titlelistall:
        item = item.strip()  # item='整租 | 大山子南里 &#x958f;&#x9e3a;&#x9f64;艺术区 &#x9f92;&#x9fa5;0;大厦 望京南地铁站单'
        titlelist_1 = item.split(';')  # ['整租 | 大山子南里 &#x958f', '&#x9e3a', '&#x9f64', '艺术区 &#x9f92', '&#x9fa5', '0', '大厦 望京南地铁站单']
        # 对单条切割好的标题信息进行编码转换处理
        for item in titlelist_1:  # 比如第一次item='整租 | 大山子南里 &#x958f'
            item = datatransform(item)  # 转换后item='整租 | 大山子南里 7'
            titlelist_2.append(item)
            titlelist_1 = []
        titleinfo = ''.join(titlelist_2)
        titlelist_real.append(titleinfo)
        titlelist_2 = []

    # 整理房子面积数据信息
    arealist = []
    arealist_real = []
    # 遍历roomlist中的元组信息，(户型信息，房子面积数据)
    for areaitem in roomlist:
        # 取出房子面积数据，用split函数返回一个列表如['&#x****','&#x****','.&#x****']
        areaitem = areaitem[1].split(';')

        # 数据信息转换，并将新数据放入新列表中
        # 新列表的变化模型应为['1',]~~['1','5']~~['1','5','.2'],
        for item in areaitem:
            item = datatransform(item)
            arealist.append(item)
        # 此时arealist=['1','5','.2'],用join方法连接
        # ps，join方法格式为    'pat'.join(list),pat默认为空
        areadata = ''.join(arealist)  # areadata=15.2，至此处理好了一个数据，一定要注意缩进，属于哪个循环作用范围
        # 数据放入新列表
        arealist_real.append(areadata)
        # 重置原列表
        arealist = []

    # 处理房子户型信息
    roomformlist_1 = []  # 存储单条标题-['主室（&#x****','）卧']
    roomformlist_2 = []  # 存储转换成数字的单条标题信息-['主室（2','）卧']
    roomformlist_real = []

    for item in roomformlist:
        item = item.strip()
        roomformlist_1 = item.split(';')
        for item in roomformlist_1:
            item = datatransform(item)
            roomformlist_2.append(item)
        roomformlist_real.append(''.join(roomformlist_2))
        roomformlist_1 = []
        roomformlist_2 = []
    roomformlist_real.pop()


# 判断当前执行是否为主程序运行，并遍历调用主函数爬取数据
def main():
    global datalist
    datalist = []  # 存放所有整合好后的信息
    url = 'http://bj.58.com/dashanzi/chuzu/pn1/?ClickID=1'
    html = getPage(url)
    analysisPage(html)
    if html:
        a = len(titlelist_real)
        for i in range(a):
            info = titlelist_real[i] + '\n' + '图片地址：' + piclist_real[i] + '\n' + '房间户型：' + roomformlist_real[i] + '      ' + '房间面积大小：' + arealist_real[i] + 'm²'
            # print(info)
            # print('=' * 40 + '分割线' + '=' * 40)
            datalist.append(info)
    # print(datalist)


if __name__ == '__main__':
    main()
    for item in datalist:
        write(item)
    #     analysisPage(html)
    # print(titlelist_real)
    # print(len(titlelist_real))
    # print(piclist_real)
    # print(len(piclist_real))
    # print(roomformlist_real)
    # print(len(roomformlist_real))
    # print(arealist_real)
    # print(len(arealist_real))
